fractal-server 1.4.9__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/app/schemas/v2/__init__.py

@@ -0,0 +1,34 @@
+from .dataset import DatasetCreateV2  # noqa F401
+from .dataset import DatasetReadV2  # noqa F401
+from .dataset import DatasetUpdateV2  # noqa F401
+from .dumps import DatasetDumpV2  # noqa F401
+from .dumps import ProjectDumpV2  # noqa F401
+from .dumps import TaskDumpV2  # noqa F401
+from .dumps import WorkflowDumpV2  # noqa F401
+from .dumps import WorkflowTaskDumpV2  # noqa F401
+from .job import JobCreateV2  # noqa F401
+from .job import JobReadV2  # noqa F401
+from .job import JobStatusTypeV2  # noqa F401
+from .job import JobUpdateV2  # noqa F401
+from .manifest import ManifestV2  # noqa F401
+from .project import ProjectCreateV2  # noqa F401
+from .project import ProjectReadV2  # noqa F401
+from .project import ProjectUpdateV2  # noqa F401
+from .task import TaskCreateV2  # noqa F401
+from .task import TaskExportV2  # noqa F401
+from .task import TaskImportV2  # noqa F401
+from .task import TaskReadV2  # noqa F401
+from .task import TaskUpdateV2  # noqa F401
+from .task_collection import TaskCollectPipV2  # noqa F401
+from .task_collection import TaskCollectStatusV2  # noqa F401
+from .workflow import WorkflowCreateV2  # noqa F401
+from .workflow import WorkflowExportV2  # noqa F401
+from .workflow import WorkflowImportV2  # noqa F401
+from .workflow import WorkflowReadV2  # noqa F401
+from .workflow import WorkflowUpdateV2  # noqa F401
+from .workflowtask import WorkflowTaskCreateV2  # noqa F401
+from .workflowtask import WorkflowTaskExportV2  # noqa F401
+from .workflowtask import WorkflowTaskImportV2  # noqa F401
+from .workflowtask import WorkflowTaskReadV2  # noqa F401
+from .workflowtask import WorkflowTaskStatusTypeV2  # noqa F401
+from .workflowtask import WorkflowTaskUpdateV2  # noqa F401

fractal_server/app/schemas/v2/dataset.py

@@ -0,0 +1,88 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import validator
+
+from .._validators import valstr
+from .._validators import valutc
+from ..v1.project import ProjectReadV1
+from .dumps import WorkflowTaskDumpV2
+from .workflowtask import WorkflowTaskStatusTypeV2
+from fractal_server.images import Filters
+
+
+class _DatasetHistoryItemV2(BaseModel):
+    """
+    Class for an item of `Dataset.history`.
+    """
+
+    workflowtask: WorkflowTaskDumpV2
+    status: WorkflowTaskStatusTypeV2
+    parallelization: Optional[dict]
+
+
+class DatasetStatusReadV2(BaseModel):
+    """
+    Response type for the
+    `/project/{project_id}/dataset/{dataset_id}/status/` endpoint
+    """
+
+    status: Optional[
+        dict[
+            str,
+            WorkflowTaskStatusTypeV2,
+        ]
+    ] = None
+
+
+# CRUD
+
+
+class DatasetCreateV2(BaseModel):
+
+    name: str
+
+    read_only: bool = False
+    zarr_dir: str
+
+    filters: Filters = Field(default_factory=Filters)
+
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+
+class DatasetReadV2(BaseModel):
+
+    id: int
+    name: str
+
+    project_id: int
+    project: ProjectReadV1
+
+    history: list[_DatasetHistoryItemV2]
+    read_only: bool
+
+    timestamp_created: datetime
+
+    zarr_dir: str
+    filters: Filters = Field(default_factory=Filters)
+
+    # Validators
+    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
+        valutc("timestamp_created")
+    )
+
+
+class DatasetUpdateV2(BaseModel):
+    class Config:
+        extra = "forbid"
+
+    name: Optional[str]
+    read_only: Optional[bool]
+    zarr_dir: Optional[str]
+    filters: Optional[Filters]
+
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))

fractal_server/app/schemas/v2/dumps.py

@@ -0,0 +1,87 @@
+"""
+
+Dump models differ from their Read counterpart in that:
+* They are directly JSON-able, without any additional encoder.
+* They may only include a subset of the Read attributes.
+
+These models are used in at least two situations:
+1. In the "*_dump" attributes of ApplyWorkflow models;
+2. In the `_DatasetHistoryItem.workflowtask` model, to trim its size.
+"""
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Extra
+from pydantic import root_validator
+
+from fractal_server.app.schemas.v1.dumps import TaskDumpV1
+from fractal_server.images import Filters
+
+
+class ProjectDumpV2(BaseModel, extra=Extra.forbid):
+
+    id: int
+    name: str
+    read_only: bool
+    timestamp_created: str
+
+
+class TaskDumpV2(BaseModel):
+    id: int
+    name: str
+    type: str
+
+    command_non_parallel: Optional[str]
+    command_parallel: Optional[str]
+    source: str
+    owner: Optional[str]
+    version: Optional[str]
+
+    input_types: dict[str, bool]
+    output_types: dict[str, bool]
+
+
+class WorkflowTaskDumpV2(BaseModel):
+    id: int
+    workflow_id: int
+    order: Optional[int]
+
+    input_filters: Filters
+
+    task_id: Optional[int]
+    task: Optional[TaskDumpV2]
+    task_legacy_id: Optional[int]
+    task_legacy: Optional[TaskDumpV1]
+
+    # Validators
+    @root_validator
+    def task_v1_or_v2(cls, values):
+        v1 = values.get("task_legacy_id")
+        v2 = values.get("task_id")
+        if ((v1 is not None) and (v2 is not None)) or (
+            (v1 is None) and (v2 is None)
+        ):
+            message = "both" if (v1 and v2) else "none"
+            raise ValueError(
+                "One and only one must be provided between "
+                f"'task_legacy_id' and 'task_id' (you provided {message})"
+            )
+        return values
+
+
+class WorkflowDumpV2(BaseModel):
+    id: int
+    name: str
+    project_id: int
+    timestamp_created: str
+
+
+class DatasetDumpV2(BaseModel):
+    id: int
+    name: str
+    project_id: int
+    read_only: bool
+    timestamp_created: str
+
+    zarr_dir: str
+    filters: Filters

fractal_server/app/schemas/v2/job.py

@@ -0,0 +1,113 @@
+from datetime import datetime
+from enum import Enum
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import validator
+from pydantic.types import StrictStr
+
+from .._validators import valstr
+from .._validators import valutc
+from .dumps import DatasetDumpV2
+from .dumps import ProjectDumpV2
+from .dumps import WorkflowDumpV2
+
+
+class JobStatusTypeV2(str, Enum):
+    """
+    Define the available job statuses
+
+    Attributes:
+        SUBMITTED:
+            The job was created. This does not guarantee that it was also
+            submitted to an executor (e.g. other errors could have prevented
+            this), nor that it is actually running (e.g. SLURM jobs could be
+            still in the queue).
+        DONE:
+            The job successfully reached its end.
+        FAILED:
+            The workflow terminated with an error.
+    """
+
+    SUBMITTED = "submitted"
+    DONE = "done"
+    FAILED = "failed"
+
+
+class JobCreateV2(BaseModel):
+
+    first_task_index: Optional[int] = None
+    last_task_index: Optional[int] = None
+    slurm_account: Optional[StrictStr] = None
+    worker_init: Optional[str]
+
+    # Validators
+    _worker_init = validator("worker_init", allow_reuse=True)(
+        valstr("worker_init")
+    )
+
+    @validator("first_task_index", always=True)
+    def first_task_index_non_negative(cls, v, values):
+        """
+        Check that `first_task_index` is non-negative.
+        """
+        if v is not None and v < 0:
+            raise ValueError(
+                f"first_task_index cannot be negative (given: {v})"
+            )
+        return v
+
+    @validator("last_task_index", always=True)
+    def first_last_task_indices(cls, v, values):
+        """
+        Check that `last_task_index` is non-negative, and that it is not
+        smaller than `first_task_index`.
+        """
+        if v is not None and v < 0:
+            raise ValueError(
+                f"last_task_index cannot be negative (given: {v})"
+            )
+
+        first_task_index = values.get("first_task_index")
+        last_task_index = v
+        if first_task_index is not None and last_task_index is not None:
+            if first_task_index > last_task_index:
+                raise ValueError(
+                    f"{first_task_index=} cannot be larger than "
+                    f"{last_task_index=}"
+                )
+        return v
+
+
+class JobReadV2(BaseModel):
+
+    id: int
+    project_id: Optional[int]
+    project_dump: ProjectDumpV2
+    user_email: str
+    slurm_account: Optional[str]
+    workflow_id: Optional[int]
+    workflow_dump: WorkflowDumpV2
+    dataset_id: Optional[int]
+    dataset_dump: DatasetDumpV2
+    start_timestamp: datetime
+    end_timestamp: Optional[datetime]
+    status: str
+    log: Optional[str]
+    working_dir: Optional[str]
+    working_dir_user: Optional[str]
+    first_task_index: Optional[int]
+    last_task_index: Optional[int]
+    worker_init: Optional[str]
+
+    _start_timestamp = validator("start_timestamp", allow_reuse=True)(
+        valutc("start_timestamp")
+    )
+    _end_timestamp = validator("end_timestamp", allow_reuse=True)(
+        valutc("end_timestamp")
+    )
+
+
+class JobUpdateV2(BaseModel):
+
+    status: JobStatusTypeV2

fractal_server/app/schemas/v2/manifest.py

@@ -0,0 +1,109 @@
+from typing import Any
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import HttpUrl
+from pydantic import root_validator
+from pydantic import validator
+
+
+class TaskManifestV2(BaseModel):
+    """
+    Represents a task within a V2 manifest.
+
+    Attributes:
+        name:
+            The task name
+        executable:
+            Path to the executable relative to the package root
+
+            Note: by package root we mean "as it will be installed". If a
+            package `Pkg` installs in the folder `pkg` the executable
+            `pkg/executable.py`, this attribute must contain only
+            `executable.py`.
+        input_type:
+            The input type accepted by the task
+        output_type:
+            The output type returned by the task
+        meta:
+            Additional information about the package, such as hash of the
+            executable, specific runtime requirements (e.g., need_gpu=True),
+            etc.
+        args_schema:
+            JSON Schema for task arguments
+        docs_info:
+            Additional information about the Task, coming from the docstring.
+        docs_link:
+            Link to Task docs.
+    """
+
+    name: str
+    executable_non_parallel: Optional[str] = None
+    executable_parallel: Optional[str] = None
+    input_types: dict[str, bool] = Field(default_factory=dict)
+    output_types: dict[str, bool] = Field(default_factory=dict)
+    meta_parallel: dict[str, Any] = Field(default_factory=dict)
+    meta_non_parallel: dict[str, Any] = Field(default_factory=dict)
+    args_schema_non_parallel: Optional[dict[str, Any]] = None
+    args_schema_parallel: Optional[dict[str, Any]] = None
+    docs_info: Optional[str] = None
+    docs_link: Optional[HttpUrl] = None
+
+
+class ManifestV2(BaseModel):
+    """
+    Packages containing tasks are required to include a special file
+    `__FRACTAL_MANIFEST__.json` in order to be discovered and used by Fractal.
+
+    This model class and the model classes it depends on provide the base
+    schema to read, write and validate manifests.
+
+    Attributes:
+        manifest_version:
+            A version string that provides indication for compatibility between
+            manifests as the schema evolves. This is for instance used by
+            Fractal to determine which subclass of the present base class needs
+            be used to read and validate the input.
+        task_list : list[TaskManifestType]
+            The list of tasks, represented as specified by subclasses of the
+            _TaskManifestBase (a.k.a. TaskManifestType)
+        has_args_schemas:
+            `True` if the manifest incldues JSON Schemas for the arguments of
+            each task.
+        args_schema_version:
+            Label of how `args_schema`s were generated (e.g. `pydantic_v1`).
+    """
+
+    manifest_version: str
+    task_list: list[TaskManifestV2]
+    has_args_schemas: bool = False
+    args_schema_version: Optional[str]
+
+    @root_validator()
+    def _check_args_schemas_are_present(cls, values):
+        has_args_schemas = values["has_args_schemas"]
+        task_list = values["task_list"]
+        if has_args_schemas is True:
+            for task in task_list:
+                if task.executable_parallel is not None:
+                    if task.args_schema_parallel is None:
+                        raise ValueError(
+                            f"Manifest has {has_args_schemas=}, but "
+                            f"task '{task.name}' has "
+                            f"{task.args_schema_parallel=}."
+                        )
+                if task.executable_non_parallel is not None:
+                    if task.args_schema_non_parallel is None:
+                        raise ValueError(
+                            f"Manifest has {has_args_schemas=}, but "
+                            f"task '{task.name}' has "
+                            f"{task.args_schema_non_parallel=}."
+                        )
+        return values
+
+    @validator("manifest_version")
+    def manifest_version_2(cls, value):
+        if value != "2":
+            raise ValueError(f"Wrong manifest version (given {value})")
+        return value

fractal_server/app/schemas/v2/project.py

@@ -0,0 +1,36 @@
+from datetime import datetime
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import validator
+
+from .._validators import valstr
+from .._validators import valutc
+
+
+class ProjectCreateV2(BaseModel):
+
+    name: str
+    read_only: bool = False
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+
+
+class ProjectReadV2(BaseModel):
+
+    id: int
+    name: str
+    read_only: bool
+    timestamp_created: datetime
+    # Validators
+    _timestamp_created = validator("timestamp_created", allow_reuse=True)(
+        valutc("timestamp_created")
+    )
+
+
+class ProjectUpdateV2(BaseModel):
+
+    name: Optional[str]
+    read_only: Optional[bool]
+    # Validators
+    _name = validator("name", allow_reuse=True)(valstr("name"))

fractal_server/app/schemas/v2/task.py

@@ -0,0 +1,121 @@
+from typing import Any
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import HttpUrl
+from pydantic import root_validator
+from pydantic import validator
+
+from .._validators import valstr
+
+
+class TaskCreateV2(BaseModel):
+    class Config:
+        extra = "forbid"
+
+    name: str
+
+    command_non_parallel: Optional[str]
+    command_parallel: Optional[str]
+    source: str
+
+    meta_parallel: Optional[dict[str, Any]]
+    meta_non_parallel: Optional[dict[str, Any]]
+    version: Optional[str]
+    args_schema_non_parallel: Optional[dict[str, Any]]
+    args_schema_parallel: Optional[dict[str, Any]]
+    args_schema_version: Optional[str]
+    docs_info: Optional[str]
+    docs_link: Optional[HttpUrl]
+
+    input_types: dict[str, bool] = Field(default={})
+    output_types: dict[str, bool] = Field(default={})
+
+    # Validators
+    @root_validator
+    def validate_commands(cls, values):
+        command_parallel = values.get("command_parallel")
+        command_non_parallel = values.get("command_non_parallel")
+        if (command_parallel is None) and (command_non_parallel is None):
+            raise ValueError(
+                "Task must have at least one valid command "
+                "(parallel and/or non_parallel)"
+            )
+        return values
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+    _command_non_parallel = validator(
+        "command_non_parallel", allow_reuse=True
+    )(valstr("command_non_parallel"))
+    _command_parallel = validator("command_parallel", allow_reuse=True)(
+        valstr("command_parallel")
+    )
+    _source = validator("source", allow_reuse=True)(valstr("source"))
+    _version = validator("version", allow_reuse=True)(valstr("version"))
+    _args_schema_version = validator("args_schema_version", allow_reuse=True)(
+        valstr("args_schema_version")
+    )
+
+
+class TaskReadV2(BaseModel):
+
+    id: int
+    name: str
+    type: Literal["parallel", "non_parallel", "compound"]
+    source: str
+    owner: Optional[str]
+    version: Optional[str]
+
+    command_non_parallel: Optional[str]
+    command_parallel: Optional[str]
+    meta_parallel: dict[str, Any]
+    meta_non_parallel: dict[str, Any]
+    args_schema_non_parallel: Optional[dict[str, Any]] = None
+    args_schema_parallel: Optional[dict[str, Any]] = None
+    args_schema_version: Optional[str]
+    docs_info: Optional[str]
+    docs_link: Optional[HttpUrl]
+    input_types: dict[str, bool]
+    output_types: dict[str, bool]
+
+
+class TaskUpdateV2(BaseModel):
+
+    name: Optional[str]
+    version: Optional[str]
+    command_parallel: Optional[str]
+    command_non_parallel: Optional[str]
+    input_types: Optional[dict[str, bool]]
+    output_types: Optional[dict[str, bool]]
+
+    # Validators
+    @validator("input_types", "output_types")
+    def val_is_dict(cls, v):
+        if not isinstance(v, dict):
+            raise ValueError
+        return v
+
+    _name = validator("name", allow_reuse=True)(valstr("name"))
+    _version = validator("version", allow_reuse=True)(
+        valstr("version", accept_none=True)
+    )
+    _command_parallel = validator("command_parallel", allow_reuse=True)(
+        valstr("command_parallel")
+    )
+    _command_non_parallel = validator(
+        "command_non_parallel", allow_reuse=True
+    )(valstr("command_non_parallel"))
+
+
+class TaskImportV2(BaseModel):
+
+    source: str
+    _source = validator("source", allow_reuse=True)(valstr("source"))
+
+
+class TaskExportV2(BaseModel):
+
+    source: str
+    _source = validator("source", allow_reuse=True)(valstr("source"))

fractal_server/app/schemas/v2/task_collection.py

@@ -0,0 +1,105 @@
+from pathlib import Path
+from typing import Literal
+from typing import Optional
+
+from pydantic import BaseModel
+from pydantic import Field
+from pydantic import validator
+
+from .._validators import valstr
+from .task import TaskReadV2
+
+
+class TaskCollectPipV2(BaseModel):
+    """
+    TaskCollectPipV2 class
+
+    This class only encodes the attributes required to trigger a
+    task-collection operation. Other attributes (that are assigned *during*
+    task collection) are defined as part of fractal-server.
+
+    Two cases are supported:
+
+        1. `package` is the path of a local wheel file;
+        2. `package` is the name of a package that can be installed via `pip`.
+
+
+    Attributes:
+        package:
+            The name of a `pip`-installable package, or the path to a local
+            wheel file.
+        package_version: Version of the package
+        package_extras: Package extras to include in the `pip install` command
+        python_version: Python version to install and run the package tasks
+        pinned_package_versions:
+            dictionary 'package':'version' used to pin versions for specific
+            packages.
+
+    """
+
+    package: str
+    package_version: Optional[str] = None
+    package_extras: Optional[str] = None
+    python_version: Optional[str] = None
+    pinned_package_versions: Optional[dict[str, str]] = None
+
+    _package_extras = validator("package_extras", allow_reuse=True)(
+        valstr("package_extras")
+    )
+    _python_version = validator("python_version", allow_reuse=True)(
+        valstr("python_version")
+    )
+
+    @validator("package")
+    def package_validator(cls, value):
+        if "/" in value:
+            if not value.endswith(".whl"):
+                raise ValueError(
+                    "Local-package path must be a wheel file "
+                    f"(given {value})."
+                )
+            if not Path(value).is_absolute():
+                raise ValueError(
+                    f"Local-package path must be absolute: (given {value})."
+                )
+        return value
+
+    @validator("package_version")
+    def package_version_validator(cls, v, values):
+
+        valstr("package_version")(v)
+
+        if values["package"].endswith(".whl"):
+            raise ValueError(
+                "Cannot provide version when package is a Wheel file."
+            )
+        return v
+
+
+class TaskCollectStatusV2(BaseModel):
+    """
+    TaskCollectStatus class
+
+    Attributes:
+        status:
+        package:
+        venv_path:
+        task_list:
+        log:
+        info:
+    """
+
+    status: Literal["pending", "installing", "collecting", "fail", "OK"]
+    package: str
+    venv_path: Path
+    task_list: Optional[list[TaskReadV2]] = Field(default=[])
+    log: Optional[str]
+    info: Optional[str]
+
+    def sanitised_dict(self):
+        """
+        Return `self.dict()` after casting `self.venv_path` to a string
+        """
+        d = self.dict()
+        d["venv_path"] = str(self.venv_path)
+        return d