fractal-server 1.4.2a5__py3-none-any.whl → 1.4.3a1__py3-none-any.whl

fractal_server/__init__.py

@@ -1 +1 @@
-__VERSION__ = "1.4.2a5"
+__VERSION__ = "1.4.3a1"

fractal_server/app/db/__init__.py

@@ -21,6 +21,12 @@ from ...syringe import Inject
 print(__name__)
 logger = set_logger(__name__)
 
+SQLITE_WARNING_MESSAGE = (
+    "SQLite is supported (for version >=3.37) but discouraged in production. "
+    "Given its partial support for ForeignKey constraints, "
+    "database consistency cannot be guaranteed."
+)
+
 
 class DB:
     """
@@ -32,7 +38,7 @@ class DB:
         try:
             return cls._engine_async
         except AttributeError:
-            cls.set_db()
+            cls.set_async_db()
             return cls._engine_async
 
     @classmethod
@@ -40,32 +46,22 @@ class DB:
         try:
             return cls._engine_sync
         except AttributeError:
-            cls.set_db()
+            cls.set_sync_db()
             return cls._engine_sync
 
     @classmethod
-    def set_db(cls):
+    def set_async_db(cls):
         settings = Inject(get_settings)
         settings.check_db()
 
         if settings.DB_ENGINE == "sqlite":
-            logger.warning(
-                "SQLite is supported (for version >=3.37) but discouraged "
-                "in production. Given its partial support for ForeignKey "
-                "constraints, database consistency cannot be guaranteed."
-            )
-
+            logger.warning(SQLITE_WARNING_MESSAGE)
             # Set some sqlite-specific options
             engine_kwargs_async = dict(poolclass=StaticPool)
-            engine_kwargs_sync = dict(
-                poolclass=StaticPool,
-                connect_args={"check_same_thread": False},
-            )
         else:
             engine_kwargs_async = {
                 "pool_pre_ping": True,
             }
-            engine_kwargs_sync = {}
 
         cls._engine_async = create_async_engine(
             settings.DATABASE_URL,
@@ -73,13 +69,6 @@ class DB:
             future=True,
             **engine_kwargs_async,
         )
-        cls._engine_sync = create_engine(
-            settings.DATABASE_SYNC_URL,
-            echo=settings.DB_ECHO,
-            future=True,
-            **engine_kwargs_sync,
-        )
-
         cls._async_session_maker = sessionmaker(
             cls._engine_async,
             class_=AsyncSession,
@@ -87,6 +76,28 @@ class DB:
             future=True,
         )
 
+    @classmethod
+    def set_sync_db(cls):
+        settings = Inject(get_settings)
+        settings.check_db()
+
+        if settings.DB_ENGINE == "sqlite":
+            logger.warning(SQLITE_WARNING_MESSAGE)
+            # Set some sqlite-specific options
+            engine_kwargs_sync = dict(
+                poolclass=StaticPool,
+                connect_args={"check_same_thread": False},
+            )
+        else:
+            engine_kwargs_sync = {}
+
+        cls._engine_sync = create_engine(
+            settings.DATABASE_SYNC_URL,
+            echo=settings.DB_ECHO,
+            future=True,
+            **engine_kwargs_sync,
+        )
+
         cls._sync_session_maker = sessionmaker(
             bind=cls._engine_sync,
             autocommit=False,
@@ -102,14 +113,14 @@ class DB:
                 cursor.close()
 
     @classmethod
-    async def get_db(cls) -> AsyncGenerator[AsyncSession, None]:
+    async def get_async_db(cls) -> AsyncGenerator[AsyncSession, None]:
         """
         Get async database session
         """
         try:
             session_maker = cls._async_session_maker()
         except AttributeError:
-            cls.set_db()
+            cls.set_async_db()
             session_maker = cls._async_session_maker()
         async with session_maker as async_session:
             yield async_session
@@ -122,11 +133,11 @@ class DB:
         try:
             session_maker = cls._sync_session_maker()
         except AttributeError:
-            cls.set_db()
+            cls.set_sync_db()
             session_maker = cls._sync_session_maker()
         with session_maker as sync_session:
             yield sync_session
 
 
-get_db = DB.get_db
+get_async_db = DB.get_async_db
 get_sync_db = DB.get_sync_db

fractal_server/app/models/dataset.py

@@ -1,13 +1,16 @@
+from datetime import datetime
 from typing import Any
 from typing import Optional
 
 from sqlalchemy import Column
 from sqlalchemy.ext.orderinglist import ordering_list
+from sqlalchemy.types import DateTime
 from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
+from ...utils import get_timestamp
 from ..schemas.dataset import _DatasetBase
 from ..schemas.dataset import _ResourceBase
 
@@ -55,6 +58,11 @@ class Dataset(_DatasetBase, SQLModel, table=True):
         sa_column=Column(JSON, server_default="[]", nullable=False)
     )
 
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
+
     class Config:
         arbitrary_types_allowed = True
 

fractal_server/app/models/workflow.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import Any
 from typing import Optional
 from typing import Union
@@ -5,12 +6,13 @@ from typing import Union
 from pydantic import validator
 from sqlalchemy import Column
 from sqlalchemy.ext.orderinglist import ordering_list
+from sqlalchemy.types import DateTime
 from sqlalchemy.types import JSON
 from sqlmodel import Field
 from sqlmodel import Relationship
 from sqlmodel import SQLModel
 
-from ..db import AsyncSession
+from ...utils import get_timestamp
 from ..schemas.workflow import _WorkflowBase
 from ..schemas.workflow import _WorkflowTaskBase
 from .task import Task
@@ -49,8 +51,8 @@ class WorkflowTask(_WorkflowTaskBase, SQLModel, table=True):
 
     id: Optional[int] = Field(default=None, primary_key=True)
 
-    workflow_id: Optional[int] = Field(foreign_key="workflow.id")
-    task_id: Optional[int] = Field(foreign_key="task.id")
+    workflow_id: int = Field(foreign_key="workflow.id")
+    task_id: int = Field(foreign_key="task.id")
     order: Optional[int]
     meta: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
     args: Optional[dict[str, Any]] = Field(sa_column=Column(JSON))
@@ -117,58 +119,10 @@ class Workflow(_WorkflowBase, SQLModel, table=True):
             cascade="all, delete-orphan",
         ),
     )
-
-    async def insert_task(
-        self,
-        task_id: int,
-        *,
-        args: Optional[dict[str, Any]] = None,
-        meta: Optional[dict[str, Any]] = None,
-        order: Optional[int] = None,
-        db: AsyncSession,
-        commit: bool = True,
-    ) -> WorkflowTask:
-        """
-        Insert a new WorkflowTask into Workflow.task_list
-
-        Args:
-            task_id: TBD
-            args: TBD
-            meta: TBD
-            order: TBD
-            db: TBD
-            commit: TBD
-        """
-        if order is None:
-            order = len(self.task_list)
-
-        # Get task from db, and extract default arguments via a Task property
-        # method
-        db_task = await db.get(Task, task_id)
-        default_args = db_task.default_args_from_args_schema
-        # Override default_args with args
-        actual_args = default_args.copy()
-        if args is not None:
-            for k, v in args.items():
-                actual_args[k] = v
-        if not actual_args:
-            actual_args = None
-
-        # Combine meta (higher priority) and db_task.meta (lower priority)
-        wt_meta = (db_task.meta or {}).copy()
-        wt_meta.update(meta or {})
-        if not wt_meta:
-            wt_meta = None
-
-        # Create DB entry
-        wf_task = WorkflowTask(task_id=task_id, args=actual_args, meta=wt_meta)
-        db.add(wf_task)
-        self.task_list.insert(order, wf_task)
-        self.task_list.reorder()  # type: ignore
-        if commit:
-            await db.commit()
-            await db.refresh(wf_task)
-        return wf_task
+    timestamp_created: datetime = Field(
+        default_factory=get_timestamp,
+        sa_column=Column(DateTime(timezone=True), nullable=False),
+    )
 
     @property
     def input_type(self):

fractal_server/app/routes/admin.py

@@ -16,7 +16,7 @@ from sqlmodel import select
 
 from ...utils import get_timestamp
 from ..db import AsyncSession
-from ..db import get_db
+from ..db import get_async_db
 from ..models import ApplyWorkflow
 from ..models import Dataset
 from ..models import JobStatusType
@@ -41,7 +41,7 @@ async def view_project(
     id: Optional[int] = None,
     user_id: Optional[int] = None,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[ProjectRead]:
     """
     Query `project` table.
@@ -73,7 +73,7 @@ async def view_workflow(
     project_id: Optional[int] = None,
     name_contains: Optional[str] = None,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[WorkflowRead]:
     """
     Query `workflow` table.
@@ -115,7 +115,7 @@ async def view_dataset(
     name_contains: Optional[str] = None,
     type: Optional[str] = None,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[DatasetRead]:
     """
     Query `dataset` table.
@@ -166,7 +166,7 @@ async def view_job(
     end_timestamp_min: Optional[datetime] = None,
     end_timestamp_max: Optional[datetime] = None,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[ApplyWorkflowRead]:
     """
     Query `ApplyWorkflow` table.
@@ -231,7 +231,7 @@ async def update_job(
     job_update: ApplyWorkflowUpdate,
     job_id: int,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[ApplyWorkflowRead]:
     """
     Change the status of an existing job.
@@ -264,7 +264,7 @@ async def update_job(
 async def stop_job(
     job_id: int,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
     Stop execution of a workflow job.
@@ -293,7 +293,7 @@ async def stop_job(
 async def download_job_logs(
     job_id: int,
     user: User = Depends(current_active_superuser),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> StreamingResponse:
     """
     Download job folder

fractal_server/app/routes/api/v1/_aux_functions.py

@@ -1,7 +1,9 @@
 """
 Auxiliary functions to get object from the database or perform simple checks
 """
+from typing import Any
 from typing import Literal
+from typing import Optional
 from typing import Union
 
 from fastapi import HTTPException
@@ -366,15 +368,72 @@ async def _get_task_check_owner(
     return task
 
 
-def _get_active_jobs_statement() -> SelectOfScalar:
+def _get_submitted_jobs_statement() -> SelectOfScalar:
     """
     Returns:
         A sqlmodel statement that selects all `ApplyWorkflow`s with
-        `ApplyWorkflow.status` equal to `submitted` or `running`.
+        `ApplyWorkflow.status` equal to `submitted`.
     """
     stm = select(ApplyWorkflow).where(
-        ApplyWorkflow.status.in_(
-            [JobStatusType.SUBMITTED, JobStatusType.RUNNING]
-        )
+        ApplyWorkflow.status == JobStatusType.SUBMITTED
     )
     return stm
+
+
+async def _workflow_insert_task(
+    *,
+    workflow_id: int,
+    task_id: int,
+    args: Optional[dict[str, Any]] = None,
+    meta: Optional[dict[str, Any]] = None,
+    order: Optional[int] = None,
+    db: AsyncSession,
+) -> WorkflowTask:
+    """
+    Insert a new WorkflowTask into Workflow.task_list
+
+    Args:
+        task_id: TBD
+        args: TBD
+        meta: TBD
+        order: TBD
+        db: TBD
+        commit: TBD
+    """
+    db_workflow = await db.get(Workflow, workflow_id)
+    if db_workflow is None:
+        raise ValueError(f"Workflow {workflow_id} does not exist")
+
+    if order is None:
+        order = len(db_workflow.task_list)
+
+    # Get task from db, and extract default arguments via a Task property
+    # method
+    db_task = await db.get(Task, task_id)
+    if db_task is None:
+        raise ValueError(f"Task {task_id} does not exist")
+
+    default_args = db_task.default_args_from_args_schema
+    # Override default_args with args
+    actual_args = default_args.copy()
+    if args is not None:
+        for k, v in args.items():
+            actual_args[k] = v
+    if not actual_args:
+        actual_args = None
+
+    # Combine meta (higher priority) and db_task.meta (lower priority)
+    wt_meta = (db_task.meta or {}).copy()
+    wt_meta.update(meta or {})
+    if not wt_meta:
+        wt_meta = None
+
+    # Create DB entry
+    wf_task = WorkflowTask(task_id=task_id, args=actual_args, meta=wt_meta)
+    db.add(wf_task)
+    db_workflow.task_list.insert(order, wf_task)
+    db_workflow.task_list.reorder()  # type: ignore
+    await db.commit()
+    await db.refresh(wf_task)
+
+    return wf_task

fractal_server/app/routes/api/v1/dataset.py

@@ -11,7 +11,7 @@ from sqlmodel import or_
 from sqlmodel import select
 
 from ....db import AsyncSession
-from ....db import get_db
+from ....db import get_async_db
 from ....models import ApplyWorkflow
 from ....models import Dataset
 from ....models import Project
@@ -28,9 +28,9 @@ from ....schemas import WorkflowExport
 from ....schemas import WorkflowTaskExport
 from ....security import current_active_user
 from ....security import User
-from ._aux_functions import _get_active_jobs_statement
 from ._aux_functions import _get_dataset_check_owner
 from ._aux_functions import _get_project_check_owner
+from ._aux_functions import _get_submitted_jobs_statement
 from ._aux_functions import _get_workflow_check_owner
 
 
@@ -46,7 +46,7 @@ async def create_dataset(
     project_id: int,
     dataset: DatasetCreate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[DatasetRead]:
     """
     Add new dataset to current project
@@ -70,7 +70,7 @@ async def create_dataset(
 async def read_dataset_list(
     project_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[DatasetRead]]:
     """
     Get dataset list for given project
@@ -96,7 +96,7 @@ async def read_dataset(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[DatasetRead]:
     """
     Get info on a dataset associated to the current project
@@ -121,7 +121,7 @@ async def update_dataset(
     dataset_id: int,
     dataset_update: DatasetUpdate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[DatasetRead]:
     """
     Edit a dataset associated to the current project
@@ -158,7 +158,7 @@ async def delete_dataset(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
     Delete a dataset associated to the current project
@@ -171,9 +171,9 @@ async def delete_dataset(
     )
     dataset = output["dataset"]
 
-    # Fail if there exist jobs that are active (that is, pending or running)
-    # and in relation with the current dataset.
-    stm = _get_active_jobs_statement().where(
+    # Fail if there exist jobs that are submitted and in relation with the
+    # current dataset.
+    stm = _get_submitted_jobs_statement().where(
         or_(
             ApplyWorkflow.input_dataset_id == dataset_id,
             ApplyWorkflow.output_dataset_id == dataset_id,
@@ -231,7 +231,7 @@ async def create_resource(
     dataset_id: int,
     resource: ResourceCreate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[ResourceRead]:
     """
     Add resource to an existing dataset
@@ -259,7 +259,7 @@ async def get_resource_list(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[ResourceRead]]:
     """
     Get resources from a dataset
@@ -287,7 +287,7 @@ async def update_resource(
     resource_id: int,
     resource_update: ResourceUpdate,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[ResourceRead]:
     """
     Edit a resource of a dataset
@@ -327,7 +327,7 @@ async def delete_resource(
     dataset_id: int,
     resource_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
     Delete a resource of a dataset
@@ -360,7 +360,7 @@ async def export_history_as_workflow(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[WorkflowExport]:
     """
     Extract a reproducible workflow from the dataset history.
@@ -374,11 +374,12 @@ async def export_history_as_workflow(
     )
     dataset = output["dataset"]
 
-    # Check whether there exists an active job such that `job.output_dataset_id
-    # == dataset_id`. If at least one such job exists, then this endpoint will
-    # fail. We do not support the use case of exporting a reproducible workflow
-    # when job execution is in progress; this may change in the future.
-    stm = _get_active_jobs_statement().where(
+    # Check whether there exists a submitted job such that
+    # `job.output_dataset_id==dataset_id`.
+    # If at least one such job exists, then this endpoint will fail.
+    # We do not support the use case of exporting a reproducible workflow when
+    # job execution is in progress; this may change in the future.
+    stm = _get_submitted_jobs_statement().where(
         ApplyWorkflow.output_dataset_id == dataset_id
     )
     res = await db.execute(stm)
@@ -427,7 +428,7 @@ async def get_workflowtask_status(
     project_id: int,
     dataset_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[DatasetStatusRead]:
     """
     Extract the status of all `WorkflowTask`s that ran on a given `Dataset`.
@@ -449,7 +450,7 @@ async def get_workflowtask_status(
     # Note: see
     # https://sqlmodel.tiangolo.com/tutorial/where/#type-annotations-and-errors
     # regarding the type-ignore in this code block
-    stm = _get_active_jobs_statement().where(
+    stm = _get_submitted_jobs_statement().where(
         ApplyWorkflow.output_dataset_id == dataset_id
     )
     res = await db.execute(stm)
@@ -516,7 +517,7 @@ async def get_workflowtask_status(
 @router.get("/dataset/", response_model=list[DatasetRead])
 async def get_user_datasets(
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[DatasetRead]:
     """
     Returns all the datasets of the current user

fractal_server/app/routes/api/v1/job.py

@@ -9,7 +9,7 @@ from fastapi.responses import StreamingResponse
 from sqlmodel import select
 
 from ....db import AsyncSession
-from ....db import get_db
+from ....db import get_async_db
 from ....models import ApplyWorkflow
 from ....models import Project
 from ....schemas import ApplyWorkflowRead
@@ -29,7 +29,7 @@ router = APIRouter()
 @router.get("/job/", response_model=list[ApplyWorkflowRead])
 async def get_user_jobs(
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> list[ApplyWorkflowRead]:
     """
     Returns all the jobs of the current user
@@ -51,7 +51,7 @@ async def get_workflow_jobs(
     project_id: int,
     workflow_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[ApplyWorkflowRead]]:
     """
     Returns all the jobs related to a specific workflow
@@ -73,7 +73,7 @@ async def read_job(
     project_id: int,
     job_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[ApplyWorkflowRead]:
     """
     Return info on an existing job
@@ -99,7 +99,7 @@ async def download_job_logs(
     project_id: int,
     job_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> StreamingResponse:
     """
     Download job folder
@@ -132,7 +132,7 @@ async def download_job_logs(
 async def get_job_list(
     project_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Optional[list[ApplyWorkflowRead]]:
     """
     Get job list for given project
@@ -156,7 +156,7 @@ async def stop_job(
     project_id: int,
     job_id: int,
     user: User = Depends(current_active_user),
-    db: AsyncSession = Depends(get_db),
+    db: AsyncSession = Depends(get_async_db),
 ) -> Response:
     """
     Stop execution of a workflow job (only available for slurm backend)