fractal-server 1.4.10__py3-none-any.whl → 2.0.0a1__py3-none-any.whl

fractal_server/app/schemas/v2/manifest.py

@@ -0,0 +1,159 @@
+from typing import Any
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import HttpUrl
+from pydantic import root_validator
+from pydantic import validator
+
+
+class TaskManifestV2(BaseModel):
+    """
+    Represents a task within a V2 manifest.
+
+    Attributes:
+        name:
+            The task name
+        executable:
+            Path to the executable relative to the package root
+
+            Note: by package root we mean "as it will be installed". If a
+            package `Pkg` installs in the folder `pkg` the executable
+            `pkg/executable.py`, this attribute must contain only
+            `executable.py`.
+        input_type:
+            The input type accepted by the task
+        output_type:
+            The output type returned by the task
+        meta:
+            Additional information about the package, such as hash of the
+            executable, specific runtime requirements (e.g., need_gpu=True),
+            etc.
+        args_schema:
+            JSON Schema for task arguments
+        docs_info:
+            Additional information about the Task, coming from the docstring.
+        docs_link:
+            Link to Task docs.
+    """
+
+    name: str
+    executable_non_parallel: Optional[str] = None
+    executable_parallel: Optional[str] = None
+    input_types: dict[str, bool] = Field(default_factory=dict)
+    output_types: dict[str, bool] = Field(default_factory=dict)
+    meta_non_parallel: dict[str, Any] = Field(default_factory=dict)
+    meta_parallel: dict[str, Any] = Field(default_factory=dict)
+    args_schema_non_parallel: Optional[dict[str, Any]] = None
+    args_schema_parallel: Optional[dict[str, Any]] = None
+    docs_info: Optional[str] = None
+    docs_link: Optional[HttpUrl] = None
+
+    @root_validator
+    def validate_executable_args_meta(cls, values):
+
+        executable_non_parallel = values.get("executable_non_parallel")
+        executable_parallel = values.get("executable_parallel")
+        if (executable_non_parallel is None) and (executable_parallel is None):
+
+            raise ValueError(
+                "`TaskManifestV2.executable_non_parallel` and "
+                "`TaskManifestV2.executable_parallel` cannot be both None."
+            )
+
+        elif executable_non_parallel is None:
+
+            meta_non_parallel = values.get("meta_non_parallel")
+            if meta_non_parallel != {}:
+                raise ValueError(
+                    "`TaskManifestV2.meta_non_parallel` must be an empty dict "
+                    "if `TaskManifestV2.executable_non_parallel` is None. "
+                    f"Given: {meta_non_parallel}."
+                )
+
+            args_schema_non_parallel = values.get("args_schema_non_parallel")
+            if args_schema_non_parallel is not None:
+                raise ValueError(
+                    "`TaskManifestV2.args_schema_non_parallel` must be None "
+                    "if `TaskManifestV2.executable_non_parallel` is None. "
+                    f"Given: {args_schema_non_parallel}."
+                )
+
+        elif executable_parallel is None:
+
+            meta_parallel = values.get("meta_parallel")
+            if meta_parallel != {}:
+                raise ValueError(
+                    "`TaskManifestV2.meta_parallel` must be an empty dict if "
+                    "`TaskManifestV2.executable_parallel` is None. "
+                    f"Given: {meta_parallel}."
+                )
+
+            args_schema_parallel = values.get("args_schema_parallel")
+            if args_schema_parallel is not None:
+                raise ValueError(
+                    "`TaskManifestV2.args_schema_parallel` must be None if "
+                    "`TaskManifestV2.executable_parallel` is None. "
+                    f"Given: {args_schema_parallel}."
+                )
+
+        return values
+
+
+class ManifestV2(BaseModel):
+    """
+    Packages containing tasks are required to include a special file
+    `__FRACTAL_MANIFEST__.json` in order to be discovered and used by Fractal.
+
+    This model class and the model classes it depends on provide the base
+    schema to read, write and validate manifests.
+
+    Attributes:
+        manifest_version:
+            A version string that provides indication for compatibility between
+            manifests as the schema evolves. This is for instance used by
+            Fractal to determine which subclass of the present base class needs
+            be used to read and validate the input.
+        task_list : list[TaskManifestType]
+            The list of tasks, represented as specified by subclasses of the
+            _TaskManifestBase (a.k.a. TaskManifestType)
+        has_args_schemas:
+            `True` if the manifest incldues JSON Schemas for the arguments of
+            each task.
+        args_schema_version:
+            Label of how `args_schema`s were generated (e.g. `pydantic_v1`).
+    """
+
+    manifest_version: str
+    task_list: list[TaskManifestV2]
+    has_args_schemas: bool = False
+    args_schema_version: Optional[str]
+
+    @root_validator()
+    def _check_args_schemas_are_present(cls, values):
+        has_args_schemas = values["has_args_schemas"]
+        task_list = values["task_list"]
+        if has_args_schemas is True:
+            for task in task_list:
+                if task.executable_parallel is not None:
+                    if task.args_schema_parallel is None:
+                        raise ValueError(
+                            f"Manifest has {has_args_schemas=}, but "
+                            f"task '{task.name}' has "
+                            f"{task.args_schema_parallel=}."
+                        )
+                if task.executable_non_parallel is not None:
+                    if task.args_schema_non_parallel is None:
+                        raise ValueError(
+                            f"Manifest has {has_args_schemas=}, but "
+                            f"task '{task.name}' has "
+                            f"{task.args_schema_non_parallel=}."
+                        )
+        return values
+
+    @validator("manifest_version")
+    def manifest_version_2(cls, value):
+        if value != "2":
+            raise ValueError(f"Wrong manifest version (given {value})")
+        return value

fractal_server/app/schemas/v2/project.py

@@ -0,0 +1,37 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic import validator
+
+from .._validators import valstr
+from .._validators import valutc
+
+
+class ProjectCreateV2(BaseModel, extra=Extra.forbid):
+
+    name: str
+    read_only: bool = False
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+
+class ProjectReadV2(BaseModel):
+
+    id: int
+    name: str
+    read_only: bool
+    timestamp_created: datetime
+    # Validators
+    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
+        valutc("timestamp_created")
+    )
+
+
+class ProjectUpdateV2(BaseModel):
+
+    name: Optional[str]
+    read_only: Optional[bool]
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))

fractal_server/app/schemas/v2/task.py

@@ -0,0 +1,120 @@
+from typing import Any
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic import Field
+from pydantic import HttpUrl
+from pydantic import root_validator
+from pydantic import validator
+
+from .._validators import valstr
+
+
+class TaskCreateV2(BaseModel, extra=Extra.forbid):
+
+    name: str
+
+    command_non_parallel: Optional[str]
+    command_parallel: Optional[str]
+    source: str
+
+    meta_parallel: Optional[dict[str, Any]]
+    meta_non_parallel: Optional[dict[str, Any]]
+    version: Optional[str]
+    args_schema_non_parallel: Optional[dict[str, Any]]
+    args_schema_parallel: Optional[dict[str, Any]]
+    args_schema_version: Optional[str]
+    docs_info: Optional[str]
+    docs_link: Optional[HttpUrl]
+
+    input_types: dict[str, bool] = Field(default={})
+    output_types: dict[str, bool] = Field(default={})
+
+    # Validators
+    @root_validator
+    def validate_commands(cls, values):
+        command_parallel = values.get("command_parallel")
+        command_non_parallel = values.get("command_non_parallel")
+        if (command_parallel is None) and (command_non_parallel is None):
+            raise ValueError(
+                "Task must have at least one valid command "
+                "(parallel and/or non_parallel)"
+            )
+        return values
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+    _command_non_parallel = validator(
+        "command_non_parallel", allow_reuse=True
+    )(valstr("command_non_parallel"))
+    _command_parallel = validator("command_parallel", allow_reuse=True)(
+        valstr("command_parallel")
+    )
+    _source = validator("source", allow_reuse=True)(valstr("source"))
+    _version = validator("version", allow_reuse=True)(valstr("version"))
+    _args_schema_version = validator("args_schema_version", allow_reuse=True)(
+        valstr("args_schema_version")
+    )
+
+
+class TaskReadV2(BaseModel):
+
+    id: int
+    name: str
+    type: Literal["parallel", "non_parallel", "compound"]
+    source: str
+    owner: Optional[str]
+    version: Optional[str]
+
+    command_non_parallel: Optional[str]
+    command_parallel: Optional[str]
+    meta_parallel: dict[str, Any]
+    meta_non_parallel: dict[str, Any]
+    args_schema_non_parallel: Optional[dict[str, Any]] = None
+    args_schema_parallel: Optional[dict[str, Any]] = None
+    args_schema_version: Optional[str]
+    docs_info: Optional[str]
+    docs_link: Optional[HttpUrl]
+    input_types: dict[str, bool]
+    output_types: dict[str, bool]
+
+
+class TaskUpdateV2(BaseModel):
+
+    name: Optional[str]
+    version: Optional[str]
+    command_parallel: Optional[str]
+    command_non_parallel: Optional[str]
+    input_types: Optional[dict[str, bool]]
+    output_types: Optional[dict[str, bool]]
+
+    # Validators
+    @validator("input_types", "output_types")
+    def val_is_dict(cls, v):
+        if not isinstance(v, dict):
+            raise ValueError
+        return v
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+    _version = validator("version", allow_reuse=True)(
+        valstr("version", accept_none=True)
+    )
+    _command_parallel = validator("command_parallel", allow_reuse=True)(
+        valstr("command_parallel")
+    )
+    _command_non_parallel = validator(
+        "command_non_parallel", allow_reuse=True
+    )(valstr("command_non_parallel"))
+
+
+class TaskImportV2(BaseModel):
+
+    source: str
+    _source = validator("source", allow_reuse=True)(valstr("source"))
+
+
+class TaskExportV2(BaseModel):
+
+    source: str
+    _source = validator("source", allow_reuse=True)(valstr("source"))

fractal_server/app/schemas/v2/task_collection.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import validator
+
+from .._validators import valstr
+from .task import TaskReadV2
+
+
+class TaskCollectPipV2(BaseModel):
+    """
+    TaskCollectPipV2 class
+
+    This class only encodes the attributes required to trigger a
+    task-collection operation. Other attributes (that are assigned *during*
+    task collection) are defined as part of fractal-server.
+
+    Two cases are supported:
+
+        1. `package` is the path of a local wheel file;
+        2. `package` is the name of a package that can be installed via `pip`.
+
+
+    Attributes:
+        package:
+            The name of a `pip`-installable package, or the path to a local
+            wheel file.
+        package_version: Version of the package
+        package_extras: Package extras to include in the `pip install` command
+        python_version: Python version to install and run the package tasks
+        pinned_package_versions:
+            dictionary 'package':'version' used to pin versions for specific
+            packages.
+
+    """
+
+    package: str
+    package_version: Optional[str] = None
+    package_extras: Optional[str] = None
+    python_version: Optional[str] = None
+    pinned_package_versions: Optional[dict[str, str]] = None
+
+    _package_extras = validator("package_extras", allow_reuse=True)(
+        valstr("package_extras")
+    )
+    _python_version = validator("python_version", allow_reuse=True)(
+        valstr("python_version")
+    )
+
+    @validator("package")
+    def package_validator(cls, value):
+        if "/" in value:
+            if not value.endswith(".whl"):
+                raise ValueError(
+                    "Local-package path must be a wheel file "
+                    f"(given {value})."
+                )
+            if not Path(value).is_absolute():
+                raise ValueError(
+                    f"Local-package path must be absolute: (given {value})."
+                )
+        return value
+
+    @validator("package_version")
+    def package_version_validator(cls, v, values):
+
+        valstr("package_version")(v)
+
+        if values["package"].endswith(".whl"):
+            raise ValueError(
+                "Cannot provide version when package is a Wheel file."
+            )
+        return v
+
+
+class TaskCollectStatusV2(BaseModel):
+    """
+    TaskCollectStatus class
+
+    Attributes:
+        status:
+        package:
+        venv_path:
+        task_list:
+        log:
+        info:
+    """
+
+    status: Literal["pending", "installing", "collecting", "fail", "OK"]
+    package: str
+    venv_path: Path
+    task_list: Optional[list[TaskReadV2]] = Field(default=[])
+    log: Optional[str]
+    info: Optional[str]
+
+    def sanitised_dict(self):
+        """
+        Return `self.dict()` after casting `self.venv_path` to a string
+        """
+        d = self.dict()
+        d["venv_path"] = str(self.venv_path)
+        return d

fractal_server/app/schemas/v2/workflow.py

@@ -0,0 +1,79 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic import validator
+
+from .._validators import valstr
+from .._validators import valutc
+from ..v1.project import ProjectReadV1
+from .workflowtask import WorkflowTaskExportV2
+from .workflowtask import WorkflowTaskImportV2
+from .workflowtask import WorkflowTaskReadV2
+
+
+class WorkflowCreateV2(BaseModel, extra=Extra.forbid):
+
+    name: str
+
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+
+class WorkflowReadV2(BaseModel):
+
+    id: int
+    name: str
+    project_id: int
+    task_list: list[WorkflowTaskReadV2]
+    project: ProjectReadV1
+    timestamp_created: datetime
+
+    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
+        valutc("timestamp_created")
+    )
+
+
+class WorkflowUpdateV2(BaseModel):
+
+    name: Optional[str]
+    reordered_workflowtask_ids: Optional[list[int]]
+
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+    @validator("reordered_workflowtask_ids")
+    def check_positive_and_unique(cls, value):
+        if any(i < 0 for i in value):
+            raise ValueError("Negative `id` in `reordered_workflowtask_ids`")
+        if len(value) != len(set(value)):
+            raise ValueError("`reordered_workflowtask_ids` has repetitions")
+        return value
+
+
+class WorkflowImportV2(BaseModel):
+    """
+    Class for `Workflow` import.
+
+    Attributes:
+        task_list:
+    """
+
+    name: str
+    task_list: list[WorkflowTaskImportV2]
+
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+
+class WorkflowExportV2(BaseModel):
+    """
+    Class for `Workflow` export.
+
+    Attributes:
+        task_list:
+    """
+
+    name: str
+    task_list: list[WorkflowTaskExportV2]

fractal_server/app/schemas/v2/workflowtask.py

@@ -0,0 +1,119 @@
+from enum import Enum
+from typing import Any
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic import Field
+from pydantic import validator
+
+from .._validators import valint
+from ..v1.task import TaskExportV1
+from ..v1.task import TaskImportV1
+from ..v1.task import TaskReadV1
+from .task import TaskExportV2
+from .task import TaskImportV2
+from .task import TaskReadV2
+from fractal_server.images import Filters
+
+
+class WorkflowTaskStatusTypeV2(str, Enum):
+    """
+    Define the available values for the status of a `WorkflowTask`.
+
+    This model is used within the `Dataset.history` attribute, which is
+    constructed in the runner and then used in the API (e.g. in the
+    `api/v2/project/{project_id}/dataset/{dataset_id}/status` endpoint).
+
+    Attributes:
+        SUBMITTED: The `WorkflowTask` is part of a running job.
+        DONE: The most-recent execution of this `WorkflowTask` was successful.
+        FAILED: The most-recent execution of this `WorkflowTask` failed.
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class WorkflowTaskCreateV2(BaseModel, extra=Extra.forbid):
+
+    meta_parallel: Optional[dict[str, Any]]
+    meta_non_parallel: Optional[dict[str, Any]]
+    args_non_parallel: Optional[dict[str, Any]]
+    args_parallel: Optional[dict[str, Any]]
+    order: Optional[int]
+    input_filters: Filters = Field(default_factory=Filters)
+
+    is_legacy_task: bool = False
+
+    # Validators
+
+    _order = validator("order", allow_reuse=True)(valint("order", min_val=0))
+    # FIXME validate: if `is_legacy_task`, `args_non_parallel` must be None
+
+
+class WorkflowTaskReadV2(BaseModel):
+
+    id: int
+
+    workflow_id: int
+    order: Optional[int]
+    meta_parallel: Optional[dict[str, Any]]
+    meta_non_parallel: Optional[dict[str, Any]]
+
+    args_non_parallel: Optional[dict[str, Any]]
+    args_parallel: Optional[dict[str, Any]]
+
+    input_filters: Filters
+
+    is_legacy_task: bool
+    task_type: str
+    task_id: Optional[int]
+    task: Optional[TaskReadV2]
+    task_legacy_id: Optional[int]
+    task_legacy: Optional[TaskReadV1]
+
+
+class WorkflowTaskUpdateV2(BaseModel):
+
+    meta_parallel: Optional[dict[str, Any]]
+    meta_non_parallel: Optional[dict[str, Any]]
+    args_non_parallel: Optional[dict[str, Any]]
+    args_parallel: Optional[dict[str, Any]]
+    input_filters: Optional[Filters]
+
+    # Validators
+
+    @validator("meta_parallel", "meta_non_parallel")
+    def check_no_parallelisation_level(cls, m):
+        if "parallelization_level" in m:
+            raise ValueError(
+                "Overriding task parallelization level currently not allowed"
+            )
+        return m
+
+
+class WorkflowTaskImportV2(BaseModel):
+
+    meta_parallel: Optional[dict[str, Any]] = None
+    meta_non_parallel: Optional[dict[str, Any]] = None
+    args: Optional[dict[str, Any]] = None  # FIXME
+
+    input_filters: Optional[Filters] = None
+
+    is_legacy_task: bool = False
+    task: Optional[TaskImportV2] = None
+    task_legacy: Optional[TaskImportV1] = None
+
+
+class WorkflowTaskExportV2(BaseModel):
+
+    meta_parallel: Optional[dict[str, Any]] = None
+    meta_non_parallel: Optional[dict[str, Any]] = None
+    args: Optional[dict[str, Any]] = None  # FIXME
+    input_filters: Filters = Field(default_factory=Filters)
+
+    is_legacy_task: bool = False
+    task: Optional[TaskExportV2]
+    task_legacy: Optional[TaskExportV1]

fractal_server/config.py

@@ -390,6 +390,11 @@ class Settings(BaseSettings):
 
         info = f"FRACTAL_RUNNER_BACKEND={self.FRACTAL_RUNNER_BACKEND}"
         if self.FRACTAL_RUNNER_BACKEND == "slurm":
+
+            from fractal_server.app.runner.executors.slurm._slurm_config import (  # noqa: E501
+                load_slurm_config_file,
+            )
+
             if not self.FRACTAL_SLURM_CONFIG_FILE:
                 raise FractalConfigurationError(
                     f"Must set FRACTAL_SLURM_CONFIG_FILE when {info}"
@@ -401,10 +406,6 @@ class Settings(BaseSettings):
                         f"{self.FRACTAL_SLURM_CONFIG_FILE} not found."
                     )
 
-                from fractal_server.app.runner._slurm._slurm_config import (
-                    load_slurm_config_file,
-                )
-
                 load_slurm_config_file(self.FRACTAL_SLURM_CONFIG_FILE)
                 if not shutil.which("sbatch"):
                     raise FractalConfigurationError(

fractal_server/images/__init__.py

@@ -0,0 +1,2 @@
+from .models import Filters  # noqa: F401
+from .models import SingleImage  # noqa: F401

fractal_server/images/models.py

@@ -0,0 +1,50 @@
+from typing import Any
+from typing import Optional
+from typing import Union
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import validator
+
+
+class SingleImage(BaseModel):
+
+    path: str
+    origin: Optional[str] = None
+
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    types: dict[str, bool] = Field(default_factory=dict)
+
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool)):
+                raise ValueError(
+                    f"SingleImage.attributes[{key}] must be a scalar "
+                    f"(int, float, str or bool). Given {value} ({type(value)})"
+                )
+        return v
+
+
+class Filters(BaseModel):
+    attributes: dict[str, Any] = Field(default_factory=dict)
+    types: dict[str, bool] = Field(default_factory=dict)
+
+    class Config:
+        extra = "forbid"
+
+    # Validators
+    @validator("attributes")
+    def validate_attributes(
+        cls, v: dict[str, Any]
+    ) -> dict[str, Union[int, float, str, bool, None]]:
+        for key, value in v.items():
+            if not isinstance(value, (int, float, str, bool, type(None))):
+                raise ValueError(
+                    f"Filters.attributes[{key}] must be a scalar "
+                    "(int, float, str, bool, or None). "
+                    f"Given {value} ({type(value)})"
+                )
+        return v