fractal-server 1.4.10__py3-none-any.whl → 2.0.0a0__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "1.4.10"
+__VERSION__ = "2.0.0a0"

fractal_server/app/models/__init__.py

@@ -1,11 +1,8 @@
 """
 `models` module
 """
-from ..schemas import *  # noqa F401
-from .dataset import *  # noqa: F403, F401
-from .job import *  # noqa: F403, F401
-from .project import *  # noqa: F403, F401
-from .security import *  # noqa: F403, F401
+from ..schemas.v1 import *  # noqa F401, F403  # FIXME: remove this
+from .security import *  # noqa: F401, F403
 from .state import State  # noqa: F401
-from .task import *  # noqa: F403, F401
-from .workflow import *  # noqa: F401, F403
+from .v1 import *  # noqa: F401, F403
+from .v2 import *  # noqa: F401, F403

fractal_server/app/models/linkuserproject.py

@@ -9,3 +9,12 @@ class LinkUserProject(SQLModel, table=True):
 
     project_id: int = Field(foreign_key="project.id", primary_key=True)
     user_id: int = Field(foreign_key="user_oauth.id", primary_key=True)
+
+
+class LinkUserProjectV2(SQLModel, table=True):
+    """
+    Crossing table between User and ProjectV2
+    """
+
+    project_id: int = Field(foreign_key="projectv2.id", primary_key=True)
+    user_id: int = Field(foreign_key="user_oauth.id", primary_key=True)

fractal_server/app/models/security.py

@@ -19,6 +19,7 @@ from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
 from .linkuserproject import LinkUserProject
+from .linkuserproject import LinkUserProjectV2
 
 
 class OAuthAccount(SQLModel, table=True):
@@ -107,6 +108,11 @@ class UserOAuth(SQLModel, table=True):
         link_model=LinkUserProject,
         sa_relationship_kwargs={"lazy": "selectin"},
     )
+    project_list_v2: list["ProjectV2"] = Relationship(  # noqa
+        back_populates="user_list",
+        link_model=LinkUserProjectV2,
+        sa_relationship_kwargs={"lazy": "selectin"},
+    )
 
     class Config:
         orm_mode = True

fractal_server/app/models/state.py

@@ -9,7 +9,7 @@ from sqlmodel import Field
 from sqlmodel import SQLModel
 
 from ...utils import get_timestamp
-from ..schemas import _StateBase
+from ..schemas.v1 import _StateBase
 
 
 class State(_StateBase, SQLModel, table=True):

fractal_server/app/models/v1/__init__.py

@@ -0,0 +1,10 @@
+"""
+`models` module
+"""
+from .dataset import Dataset  # noqa: F401
+from .dataset import Resource  # noqa: F401
+from .job import ApplyWorkflow  # noqa: F403, F401
+from .project import Project  # noqa: F403, F401
+from .task import Task  # noqa: F403, F401
+from .workflow import Workflow  # noqa: F401, F403
+from .workflow import WorkflowTask  # noqa: F401, F403

fractal_server/app/models/{dataset.py → v1/dataset.py}

@@ -10,17 +10,17 @@ from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
-from ...utils import get_timestamp
-from ..schemas.dataset import _DatasetBase
-from ..schemas.dataset import _ResourceBase
+from ....utils import get_timestamp
+from ...schemas.v1.dataset import _DatasetBaseV1
+from ...schemas.v1.dataset import _ResourceBaseV1
 
 
-class Resource(_ResourceBase, SQLModel, table=True):
+class Resource(_ResourceBaseV1, SQLModel, table=True):
     id: Optional[int] = Field(default=None, primary_key=True)
     dataset_id: int = Field(foreign_key="dataset.id")
 
 
-class Dataset(_DatasetBase, SQLModel, table=True):
+class Dataset(_DatasetBaseV1, SQLModel, table=True):
     """
     Represent a dataset
 

fractal_server/app/models/{job.py → v1/job.py}

@@ -8,12 +8,12 @@ from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
-from ...utils import get_timestamp
-from ..schemas import JobStatusType
-from ..schemas.applyworkflow import _ApplyWorkflowBase
+from ....utils import get_timestamp
+from ...schemas.v1 import JobStatusTypeV1
+from ...schemas.v1.applyworkflow import _ApplyWorkflowBaseV1
 
 
-class ApplyWorkflow(_ApplyWorkflowBase, SQLModel, table=True):
+class ApplyWorkflow(_ApplyWorkflowBaseV1, SQLModel, table=True):
     """
     Represent a workflow run
 
@@ -97,5 +97,5 @@ class ApplyWorkflow(_ApplyWorkflowBase, SQLModel, table=True):
     end_timestamp: Optional[datetime] = Field(
         default=None, sa_column=Column(DateTime(timezone=True))
     )
-    status: str = JobStatusType.SUBMITTED
+    status: str = JobStatusTypeV1.SUBMITTED
     log: Optional[str] = None

fractal_server/app/models/{project.py → v1/project.py}

@@ -7,13 +7,13 @@ from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
-from ...utils import get_timestamp
-from ..schemas.project import _ProjectBase
-from .linkuserproject import LinkUserProject
-from .security import UserOAuth
+from ....utils import get_timestamp
+from ...schemas.v1.project import _ProjectBaseV1
+from ..linkuserproject import LinkUserProject
+from ..security import UserOAuth
 
 
-class Project(_ProjectBase, SQLModel, table=True):
+class Project(_ProjectBaseV1, SQLModel, table=True):
 
     id: Optional[int] = Field(default=None, primary_key=True)
     timestamp_created: datetime = Field(

fractal_server/app/models/{task.py → v1/task.py}

@@ -5,14 +5,15 @@ from typing import Optional
 
 from pydantic import HttpUrl
 from sqlalchemy import Column
+from sqlalchemy import sql
 from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import SQLModel
 
-from ..schemas.task import _TaskBase
+from ...schemas.v1.task import _TaskBaseV1
 
 
-class Task(_TaskBase, SQLModel, table=True):
+class Task(_TaskBaseV1, SQLModel, table=True):
     """
     Task model
 
@@ -48,6 +49,10 @@ class Task(_TaskBase, SQLModel, table=True):
     docs_info: Optional[str] = None
     docs_link: Optional[HttpUrl] = None
 
+    is_v2_compatible: bool = Field(
+        default=False, sa_column_kwargs={"server_default": sql.false()}
+    )
+
     @property
     def parallelization_level(self) -> Optional[str]:
         try:

fractal_server/app/models/{workflow.py → v1/workflow.py}

@@ -12,13 +12,13 @@ from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
-from ...utils import get_timestamp
-from ..schemas.workflow import _WorkflowBase
-from ..schemas.workflow import _WorkflowTaskBase
+from ....utils import get_timestamp
+from ...schemas.v1.workflow import _WorkflowBaseV1
+from ...schemas.v1.workflow import _WorkflowTaskBaseV1
 from .task import Task
 
 
-class WorkflowTask(_WorkflowTaskBase, SQLModel, table=True):
+class WorkflowTask(_WorkflowTaskBaseV1, SQLModel, table=True):
     """
     A Task as part of a Workflow
 
@@ -92,7 +92,7 @@ class WorkflowTask(_WorkflowTaskBase, SQLModel, table=True):
         return self.task.parallelization_level
 
 
-class Workflow(_WorkflowBase, SQLModel, table=True):
+class Workflow(_WorkflowBaseV1, SQLModel, table=True):
     """
     Workflow
 

fractal_server/app/models/v2/__init__.py

@@ -0,0 +1,20 @@
+"""
+v2 `models` module
+"""
+from ..linkuserproject import LinkUserProjectV2
+from .dataset import DatasetV2
+from .job import JobV2
+from .project import ProjectV2
+from .task import TaskV2
+from .workflow import WorkflowV2
+from .workflowtask import WorkflowTaskV2
+
+__all__ = [
+    "LinkUserProjectV2",
+    "DatasetV2",
+    "JobV2",
+    "ProjectV2",
+    "TaskV2",
+    "WorkflowTaskV2",
+    "WorkflowV2",
+]

fractal_server/app/models/v2/dataset.py

@@ -0,0 +1,55 @@
+from datetime import datetime
+from typing import Any
+from typing import Literal
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.types import DateTime
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+
+
+class DatasetV2(SQLModel, table=True):
+    class Config:
+        arbitrary_types_allowed = True
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    name: str
+
+    project_id: int = Field(foreign_key="projectv2.id")
+    project: "ProjectV2" = Relationship(  # noqa: F821
+        sa_relationship_kwargs=dict(lazy="selectin"),
+    )
+
+    history: list[dict[str, Any]] = Field(
+        sa_column=Column(JSON, server_default="[]", nullable=False)
+    )
+    read_only: bool = False
+
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+
+    # New in V2
+
+    zarr_dir: str
+    images: list[dict[str, Any]] = Field(
+        sa_column=Column(JSON, server_default="[]", nullable=False)
+    )
+
+    filters: dict[Literal["attributes", "types"], dict[str, Any]] = Field(
+        sa_column=Column(
+            JSON,
+            nullable=False,
+            server_default='{"attributes": {}, "types": {}}',
+        )
+    )
+
+    @property
+    def image_paths(self) -> list[str]:
+        return [image["path"] for image in self.images]

fractal_server/app/models/v2/job.py

@@ -0,0 +1,51 @@
+from datetime import datetime
+from typing import Any
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.types import DateTime
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+from ...schemas.v1 import JobStatusTypeV1
+
+
+class JobV2(SQLModel, table=True):
+    class Config:
+        arbitrary_types_allowed = True
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    project_id: Optional[int] = Field(foreign_key="projectv2.id")
+    workflow_id: Optional[int] = Field(foreign_key="workflowv2.id")
+    dataset_id: Optional[int] = Field(foreign_key="datasetv2.id")
+
+    user_email: str = Field(nullable=False)
+    slurm_account: Optional[str]
+
+    dataset_dump: dict[str, Any] = Field(
+        sa_column=Column(JSON, nullable=False)
+    )
+    workflow_dump: dict[str, Any] = Field(
+        sa_column=Column(JSON, nullable=False)
+    )
+    project_dump: dict[str, Any] = Field(
+        sa_column=Column(JSON, nullable=False)
+    )
+
+    worker_init: Optional[str]
+    working_dir: Optional[str]
+    working_dir_user: Optional[str]
+    first_task_index: int
+    last_task_index: int
+
+    start_timestamp: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+    end_timestamp: Optional[datetime] = Field(
+        default=None, sa_column=Column(DateTime(timezone=True))
+    )
+    status: str = JobStatusTypeV1.SUBMITTED
+    log: Optional[str] = None

fractal_server/app/models/v2/project.py

@@ -0,0 +1,31 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.types import DateTime
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+from ..linkuserproject import LinkUserProjectV2
+from ..security import UserOAuth
+
+
+class ProjectV2(SQLModel, table=True):
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    name: str
+    read_only: bool = False
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+
+    user_list: list[UserOAuth] = Relationship(
+        link_model=LinkUserProjectV2,
+        back_populates="project_list_v2",
+        sa_relationship_kwargs={
+            "lazy": "selectin",
+        },
+    )

fractal_server/app/models/v2/task.py

@@ -0,0 +1,93 @@
+import json
+import logging
+from typing import Any
+from typing import Optional
+
+from pydantic import HttpUrl
+from sqlalchemy import Column
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import SQLModel
+
+
+class TaskV2(SQLModel, table=True):
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    name: str
+
+    type: str
+    command_non_parallel: Optional[str] = None
+    command_parallel: Optional[str] = None
+    source: str = Field(unique=True)
+
+    meta_non_parallel: dict[str, Any] = Field(
+        sa_column=Column(JSON, server_default="{}", default={}, nullable=False)
+    )
+    meta_parallel: dict[str, Any] = Field(
+        sa_column=Column(JSON, server_default="{}", default={}, nullable=False)
+    )
+
+    owner: Optional[str] = None
+    version: Optional[str] = None
+    args_schema_non_parallel: Optional[dict[str, Any]] = Field(
+        sa_column=Column(JSON), default=None
+    )
+    args_schema_parallel: Optional[dict[str, Any]] = Field(
+        sa_column=Column(JSON), default=None
+    )
+    args_schema_version: Optional[str]
+    docs_info: Optional[str] = None
+    docs_link: Optional[HttpUrl] = None
+
+    input_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
+    output_types: dict[str, bool] = Field(sa_column=Column(JSON), default={})
+
+    @property
+    def default_args_non_parallel_from_args_schema(self) -> dict[str, Any]:
+        """
+        Extract default arguments from args_schema
+        """
+        # Return {} if there is no args_schema
+        if self.args_schema_non_parallel is None:
+            return {}
+        # Try to construct default_args
+        try:
+            default_args = {}
+            properties = self.args_schema_non_parallel["properties"]
+            for prop_name, prop_schema in properties.items():
+                default_value = prop_schema.get("default", None)
+                if default_value is not None:
+                    default_args[prop_name] = default_value
+            return default_args
+        except KeyError as e:
+            logging.warning(
+                "Cannot set default_args from args_schema_non_parallel="
+                f"{json.dumps(self.args_schema_non_parallel)}\n"
+                f"Original KeyError: {str(e)}"
+            )
+            return {}
+
+    @property
+    def default_args_parallel_from_args_schema(self) -> dict[str, Any]:
+        """
+        Extract default arguments from args_schema
+        """
+        # Return {} if there is no args_schema
+        if self.args_schema_parallel is None:
+            return {}
+        # Try to construct default_args
+        try:
+            default_args = {}
+            properties = self.args_schema_parallel["properties"]
+            for prop_name, prop_schema in properties.items():
+                default_value = prop_schema.get("default", None)
+                if default_value is not None:
+                    default_args[prop_name] = default_value
+            return default_args
+        except KeyError as e:
+            logging.warning(
+                "Cannot set default_args from args_schema_parallel="
+                f"{json.dumps(self.args_schema_parallel)}\n"
+                f"Original KeyError: {str(e)}"
+            )
+            return {}

fractal_server/app/models/v2/workflow.py

@@ -0,0 +1,43 @@
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy import Column
+from sqlalchemy.ext.orderinglist import ordering_list
+from sqlalchemy.types import DateTime
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+
+from ....utils import get_timestamp
+from .workflowtask import WorkflowTaskV2
+
+
+class WorkflowV2(SQLModel, table=True):
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+    name: str
+    project_id: int = Field(foreign_key="projectv2.id")
+    project: "ProjectV2" = Relationship(  # noqa: F821
+        sa_relationship_kwargs=dict(lazy="selectin"),
+    )
+
+    task_list: list[WorkflowTaskV2] = Relationship(
+        sa_relationship_kwargs=dict(
+            lazy="selectin",
+            order_by="WorkflowTaskV2.order",
+            collection_class=ordering_list("order"),
+            cascade="all, delete-orphan",
+        ),
+    )
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+
+    @property
+    def input_types(self):
+        return self.task_list[0].task.input_types
+
+    @property
+    def output_types(self):
+        return self.task_list[-1].task.output_types

fractal_server/app/models/v2/workflowtask.py

@@ -0,0 +1,90 @@
+from typing import Any
+from typing import Literal
+from typing import Optional
+
+from pydantic import validator
+from sqlalchemy import Column
+from sqlalchemy.types import JSON
+from sqlmodel import Field
+from sqlmodel import Relationship
+from sqlmodel import SQLModel
+
+from ..v1.task import Task
+from .task import TaskV2
+
+
+class WorkflowTaskV2(SQLModel, table=True):
+    class Config:
+        arbitrary_types_allowed = True
+        fields = {"parent": {"exclude": True}}
+
+    id: Optional[int] = Field(default=None, primary_key=True)
+
+    workflow_id: int = Field(foreign_key="workflowv2.id")
+    order: Optional[int]
+    meta_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
+    meta_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
+    args_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
+    args_non_parallel: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
+
+    input_filters: dict[
+        Literal["attributes", "types"], dict[str, Any]
+    ] = Field(
+        sa_column=Column(
+            JSON,
+            nullable=False,
+            server_default='{"attributes": {}, "types": {}}',
+        )
+    )
+
+    # Task
+    is_legacy_task: bool
+    task_type: str
+    task_id: Optional[int] = Field(foreign_key="taskv2.id")
+    task: Optional[TaskV2] = Relationship(
+        sa_relationship_kwargs=dict(lazy="selectin")
+    )
+    task_legacy_id: Optional[int] = Field(foreign_key="task.id")
+    task_legacy: Optional[Task] = Relationship(
+        sa_relationship_kwargs=dict(lazy="selectin")
+    )
+
+    @validator("args_non_parallel")
+    def validate_args_non_parallel(cls, value):
+        """
+        FIXME V2 this requires an update
+        """
+        if value is None:
+            return
+        forbidden_args_keys = {
+            "metadata",
+            "component",
+        }
+        args_keys = set(value.keys())
+        intersect_keys = forbidden_args_keys.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value
+
+    @validator("args_parallel")
+    def validate_args_parallel(cls, value):
+        """
+        FIXME V2 this requires an update
+        """
+        if value is None:
+            return
+        forbidden_args_keys = {
+            "metadata",
+            "component",
+        }
+        args_keys = set(value.keys())
+        intersect_keys = forbidden_args_keys.intersection(args_keys)
+        if intersect_keys:
+            raise ValueError(
+                "`args` contains the following forbidden keys: "
+                f"{intersect_keys}"
+            )
+        return value